#include <vector>
#include <list>
#include <map>
#include <set>
#include <deque>
#include <queue>
#include <stack>
#include <bitset>
#include <algorithm>
#include <functional>
#include <numeric>
#include <iterator>
#include <utility>
#include <sstream>
#include <iostream>
#include <iomanip>
#include <cstdio>
#include <cmath>
#include <cstdlib>
#include <cctype>
#include <string>
#include <cstring>
#include <cstdio>
#include <cmath>
#include <cstdlib>
#include <ctime>
#include <hash_map>
#define for_size(var, size) \
    for(int var = 0; var < (size); ++var)
#define for_range(var, from, to) \
    for(int var = (from); var < (to); ++var)
#define for_ranges(var, from, to, step) \
    for(int var = (from); var < (to); var += (step))
#define for_each(var, container) \
    for(typeof((container).begin()) var = (container).begin(); var != (container).end(); ++var)

using namespace std;

typedef __gnu_cxx::hash_map<int, int> Counter;

const int TOOL_NUM = 5;
const int TOOL_ID_LDAC = 0;
const int TOOL_ID_GIBBS = 1;
const int TOOL_ID_PLDA = 2;
const int TOOL_ID_LDA = 3;
const int TOOL_ID_GRAPHLAB = 4;
const string OUT_SUFX[] = {".lda-c", ".gibbs", ".plda", ".lda", ".graphlab"};

string outFName[TOOL_NUM];
FILE* outF[TOOL_NUM];
FILE* inF;
int outputID;

string inFName;


bool willOutput(int id) {
    return outputID == -1 || id == outputID;
}
int main(int argc, char* argv[]) {
    if (argc == 1) {
        printf("Usage: %s file [mode]\n    mode: 0=lda-c, 1=gibbs, 2=plda, 3=lda, 4=graphlab\n", argv[0]);
        return 1;
    }
    inFName = argv[1];
    outputID = argc > 2 ? atoi(argv[2]) : -1;
    Counter wordC;
    inF = fopen(inFName.data(), "r");
    for_size(i, TOOL_NUM) {
        if (!willOutput(i))
            continue; 
        outFName[i] = inFName + OUT_SUFX[i];
        outF[i] = fopen(outFName[i].data(), "w");
    }
    int recordCnt = 0;
    int totalWordCnt = 0;
    int docCnt = 0;
    int docId;
    char buffer[1024];
    while (fscanf(inF, "%d%s", &docId, buffer) == 2) {
        docId++;
        docCnt++;
        replace(buffer, buffer + strlen(buffer), '|', ' ');
        istringstream iss(buffer);
        Counter curWordC;
        for (int wordId; iss >> wordId; curWordC[++wordId]++);
        if (willOutput(TOOL_ID_LDAC)) {
            fprintf(outF[TOOL_ID_LDAC], "%d", (int) curWordC.size());
        }
        for_each(it, curWordC) {
            if (willOutput(TOOL_ID_LDAC)) {
                fprintf(outF[TOOL_ID_LDAC], " %d:%d", it->first, it->second);
            }
            if (willOutput(TOOL_ID_GIBBS)) {
                for_size(i, it->second) {
                    fprintf(outF[TOOL_ID_GIBBS], "%d ", it->first);
                }
            }
            if (willOutput(TOOL_ID_PLDA)) {
                fprintf(outF[TOOL_ID_PLDA], "%d %d ", it->first, it->second);
            }
            if (willOutput(TOOL_ID_LDA)) {
                fprintf(outF[TOOL_ID_LDA], "%d:%d ", it->first, it->second);
            }
            if (willOutput(TOOL_ID_GRAPHLAB)) {
                fprintf(outF[TOOL_ID_GRAPHLAB], "%d %d %d\n", docId, it->first, it->second);
            }
            wordC[it->first] += it->second;
            totalWordCnt += it->second;
            recordCnt++;
        }
        if (willOutput(TOOL_ID_LDAC)) {
            fprintf(outF[TOOL_ID_LDAC], "\n");
        }
        if (willOutput(TOOL_ID_GIBBS)) {
            fprintf(outF[TOOL_ID_GIBBS], "\n");
        }
        if (willOutput(TOOL_ID_PLDA)) {
            fprintf(outF[TOOL_ID_PLDA], "\n");
        }
        if (willOutput(TOOL_ID_LDA)) {
            fprintf(outF[TOOL_ID_LDA], "\n");
        }
    }
    for_size(i, TOOL_NUM) {
        if (!willOutput(i))
            continue;
        fclose(outF[i]);
    }
    int wordCnt = wordC.size();
    printf("num_doc: %d; num_word: %d; num_record: %d; num_totalword: %d\n", docCnt, wordCnt, recordCnt, totalWordCnt);
    string wordFreqFName = inFName + ".wordFreq";
    FILE* freqF = fopen(wordFreqFName.data(), "w");
    for_each(it, wordC) {
        fprintf(freqF, "%d %d\n", it->first, it->second);
    }
    fclose(freqF);

    if (willOutput(TOOL_ID_GRAPHLAB)) { 
        sprintf(buffer, "echo -e \"%s%d %d %d\" | cat - %s > %s", "%%MatrixMarket matrix coordinate real general\\n", docCnt, wordCnt, recordCnt, outFName[TOOL_ID_GRAPHLAB].data(), (outFName[TOOL_ID_GRAPHLAB] + ".done").data());
        system(buffer);
        sprintf(buffer, "mv %s %s", (outFName[TOOL_ID_GRAPHLAB] + ".done").data(), outFName[TOOL_ID_GRAPHLAB].data());
        system(buffer);
    }

    if (willOutput(TOOL_ID_GIBBS)) {
        sprintf(buffer, "echo %d | cat - %s > %s", docCnt, outFName[TOOL_ID_GIBBS].data(), (outFName[TOOL_ID_GIBBS] + ".done").data());
        system(buffer);
        sprintf(buffer, "mv %s %s", (outFName[TOOL_ID_GIBBS] + ".done").data(), outFName[TOOL_ID_GIBBS].data());
        system(buffer);
    }

    return 0;
}

